import re
import math
import os
import sqlite3
from NewsMining.db import dbConnect
from mynewsweb.model import *
import orange
import orngSVM
import orngTest
import orngStat
from numpy import dot
from orngSVM import KernelWrapper
import ChoseBOW
import numpy as np
import itertools
import sys
import testData
import time

from svm import *

def tf_idf(text_word_freq, text_word_count, documents_number, documents_word_count):
    return (text_word_freq / float(text_word_count)) * math.log(documents_number / (float(documents_word_count) + 1), 2)

print "Start:", time.asctime()
out = open("results/Bow3.txt", "a")
#==== ==== ==== ====
# Settings
#==== ==== ==== ====
try:
    numTests = int(sys.argv[1])
    sample_length = int(sys.argv[2])
    count = int(sys.argv[3])
    limit = int(sys.argv[4])
    runOrange, runDB = 0, 0

except:
    print "WordSeqKernelOrange.py numTests sample_length count limit"
    numTests = 3
    sample_length = 80
    count = 13
    limit = 3500
    runOrange, runDB = 0, 0

db = dbConnect.getDBConnection()

#==== ==== ==== ====
# PREPARE DATA
#==== ==== ==== ====
svm_data = []
# KERNEL_WORDS
kernel_words = ChoseBOW.getBow(count, limit)
kernel_words_length = len(kernel_words)

print "kernel_words length", kernel_words_length

print "Get News"
all_news = testData.getAllData(sample_length, random=True)
#all_news = testData.getData(sample_length)
news = {}

print "Calculate tf-idf on all news"
for n in all_news:
    news_words = n.words

    news_words = dict([(nw.word_id, nw.count) for nw in news_words])
    news_words_length = len(news_words)
    #print "*" * 10, news_words_length
    if news_words_length > 0:
        # JOIN NEWS-WORDS WITH KERNEL-WORDS + TF-IDF
        news_kernel_words = np.array([tf_idf(news_words.get(w, 0), news_words_length, kernel_words_length, kernel_words[w]) for w in kernel_words], dtype='float32')
        news[n.news_id] = news_kernel_words

        svm_data.append((int(n.class_id), map(float, news_kernel_words)))
        #svm_samples.append(map(float, news_kernel_words))


#==== ==== ==== ====
# REAL SVM
#==== ==== ==== ====
def fMeasure(precision, recall):
    try:
        return 2 * ((precision * recall) / (precision + recall))
    except:
        return 0.0

def auc(result):
    result = sorted(result, lambda x, y: cmp(x[1][1], y[1][1]))
    for r in result:
        print r[0], "\t", r[1][0], "\t", round(r[1][1][1], 2)
    ploscina = x = y = 0

    for i in result:
        if i[0] == 1:
            x += 1
        else:
            ploscina += x
            y += 1
    try:
        return ploscina / float(x * y)
    except:
        return -1.0



prop = [0.2, 0.4, 0.8, 1.0]
C = 10
weight = [1, 4] # 1, 4 teh best
prediction_results = {}
orange_results = {}

##for C in [1, 3, 4, 5, 8, 10, 20]:
##    for posWeight in range(1, 10, 2):
for p in prop:
    #weight = [1, posWeight]
    print "#" * 20
    print "C: %s, %s" % (C, weight)
    print "#" * 20

    index = int(len(svm_data) * p * 0.6)
    indexTo = int(len(svm_data) * p)
    train_labels = []
    train_samples = []
    prediction_results[p] = []

    # ==== TRAIN DATA ====
    for i in svm_data[:index]:
        train_labels.append(i[0])
        train_samples.append(i[1])

    problem = svm_problem(train_labels, train_samples)
    param = svm_parameter(kernel_type= LINEAR,
                            C = C,
                            nr_weight = 2,
                            weight_label = [-1, 1],
                            weight = weight,
                            probability = 1)
                            #, nr_weight = 2, weight_label = [1,0], weight = [10,1],
    #param = svm_parameter(svm_type = C_SVC, kernel_type = LINEAR, probability = 1)
    model = svm_model(problem,param)

    # ==== CA ====
    correct = 0
    truePositive = 0
    myPositive = 0
    allPositive = 0
    ca = precision = recall = 0

    ex = orngTest.ExperimentResults(2, ['LIN'], [1, -1], -1)


    for i in svm_data[index:indexTo]:
        #pred = model.predict_probability(i[1])
        prediction = model.predict(i[1])
        probability = model.predict_probability(i[1])
        prediction_results[p].append((prediction, probability))
        #print i[0], pred
        # Orange
        t = orngTest.TestedExample(0, int(probability[0]))
        t.probabilities = [[probability[1][-1], probability[1][1]]]
        ex.results.append(t)
        # ca
        if i[0] == prediction:
            correct += 1
        # precision, Kolikokrat sem rekel, da ima nekdo bolezen
        if prediction == 1:
            myPositive += 1
            # Kolikokrat sem pravilno rekel, da ima nekdo bolezen
            if i[0] == prediction == 1:
                truePositive += 1
        # recall, Kolikokrat bi moral, reci, da ima nekdo bolezen
        if i[0] == 1:
            allPositive += 1

    orange_results[p] = ex

    try:
        ca = correct / float(len(svm_data[index:indexTo]))
        precision = truePositive / float(myPositive)
        recall = truePositive / float(allPositive)
    except:
        print "error" * 10

    f1 = fMeasure(precision, recall)
    auc_ = auc(prediction_results[p])
    try:
        auc_orange = orngStat.AUC(orange_results[p])[0]
        #f1_orange = orngStat.F1(orange_results[p])[0]
    except:
        auc_orange = -1.0
        #f1_orange = -1.0

    print "*" * 10
    print p
    print "CA:", ca
    print "Precision:", precision
    print "recall:", recall
    print "F1:", f1
    #print "F1 orange:", f1_orange
    print "AUC:", auc_
    print "AUC_orange:", auc_orange

    results = [round(i, 3) for i in ca, precision, recall, f1, auc_, auc_orange]
    toPrint = map(str, [count, limit, kernel_words_length, p, len(svm_data) * p, C, weight] + results)

    #print "\t".join(toPrint)
    out.write("\t".join(toPrint) + "\n")

    out.flush()

out.write("\n")

#==== ==== ==== ====
# ORANGE SVM
#==== ==== ==== ====
print "Create Orange examples"
def getWord(word_id):
    word = db.query(Word).filter(Word.word_id == word_id).one().word
    return re.sub("[^\w]", '', word.encode('utf-8'))

# Data in orange format
domainDef = [orange.EnumVariable(name="class_id", values=["1", "-1"])] #, orange.StringVariable("news_id")
domainWords = [orange.FloatVariable(getWord(w)) for w in kernel_words]
domain = orange.Domain(domainDef)

news_id_var = orange.StringVariable("news_id")
id = orange.newmetaid()
domain.addmeta(id, news_id_var)

if runOrange:
    examples = []

    for n in all_news:
        d =  [str(n.class_id)] #list(news[n.news_id]) +
        ex = orange.Example(domain, d)
        ex['news_id'] = str(n.news_id)
        examples.append(ex)

    data = orange.ExampleTable(examples)


#==== ==== ==== ====
# SVM - kernel
#==== ==== ==== ====
def bow_kernel(n1, n2, data):
    return dot(data[n1], data[n2])

def cached_kernel(kernel, maxSize=10000000):
    cache = {}
    def kernel2(x, y, data):
        if cache.has_key((x,y)):
            return cache[(x,y)]
        if cache.has_key((y,x)):
            return cache[(y,x)]
        #print "Run kernel"
        value = kernel(x, y, data)
        if len(cache) == maxSize:
            cache.popitem()
        cache[(x,y)] = value
        return value
    return kernel2

kernel = cached_kernel(bow_kernel)

def intermediate_kernel(n1, n2):
    return kernel(n1, n2, news)

def example_kernel(example1, example2):
    return intermediate_kernel(int(example1['news_id'].value), int(example2['news_id'].value))
    #d = example1.domain[:-1]
    #return dot(np.array([example1[i] for i in d]), np.array([example2[i] for i in d]))
    #return dot_product([example1[i] for i in d], [example2[i] for i in d])


print "Run SVM!!!"
if runOrange:
    # Build SVM
    l1=orngSVM.SVMLearner(weight=[1, 1])
    l1.kernelFunc=KernelWrapper(example_kernel) #orngSVM. #Linear
    l1.kernel_type=orange.SVMLearner.Custom
    l1.probability=True
    l1.name="LIN"

    l2=orngSVM.SVMLearner()
    l2.kernelFunc=orngSVM.RBFKernelWrapper(example_kernel, gamma=0.5)
    l2.kernel_type=orange.SVMLearner.Custom
    l2.probability=True
    l2.name="RBF"

    l3=orngSVM.SVMLearner()
    l3.kernelFunc=orngSVM.PolyKernelWrapper(example_kernel, degree=3.0)
    l3.kernel_type=orange.SVMLearner.Custom
    l3.probability=True
    l3.name="POLY"

    # ==== LEARNING CURVE =====
    prop = [0.1, 0.2, 0.4, 0.5, 0.6, 0.8, 1.0]
    res = orngTest.learningCurveN([l1, l2, l3][:numTests], data, folds = 2, proportions = prop)
    for i, p in enumerate(prop):

        for r in res[i].results:
            pass
            #print r.actualClass, r.probabilities[0]

        print "#" * 10

        ca, auc = orngStat.CA(res[i]), orngStat.AUC(res[i])
        confusionMatrica = orngStat.confusionMatrices(res[i])
        fMeasure, fAlpha = orngStat.F1(confusionMatrica), orngStat.Falpha(confusionMatrica, 0.5)

        results = [round(i, 3) for j in zip(ca, auc, fMeasure, fAlpha) for i in j]
        toPrint = map(str, [count, limit, kernel_words_length, p, len(news) * p] + results)


        print "\t".join(toPrint)
        out.write("\t".join(toPrint) + "\n")
        out.flush()


##    # RUN Tests, CA
##    print "... tests (crossValidation)"
##
##    tests = orngTest.crossValidation([l1, l2, l3][:numTests], data, folds=2)
##    print "--:", time.asctime()
##    results = orngStat.CA(tests)
##    auc = orngStat.AUC(tests)
##    confusionMatrica = orngStat.confusionMatrices(tests)
##    fMeasure = orngStat.F1(confusionMatrica)
##    fAlpha = orngStat.Falpha(confusionMatrica, 0.5)
##
##    print "*" * 10
##    print "SVM: BOW Tf-Idf"
##    print "Random sample size: %s" % len(news)
##    print " "
##
##    results = [i for j in zip(results, auc, fMeasure, fAlpha) for i in j]
##    results = [round(i, 3) for i in results]
##    toPrint = map(str, [count, limit, kernel_words_length, len(news)] + results)
##
##    print "\t".join(["count","limit", "words", "examples"] + [l.name for l in [l1, l2, l3][:numTests]])
##    print "\t".join(toPrint)
##    out.write("\t".join(toPrint) + "\n")


#==== ==== ==== ====
# CALCULATE DOT_PRODUCT between Examples
#==== ==== ==== ====
if runDB:
    print "Clear previus results"
    db.execute("delete from  kernel_results where kernel_id = 0")
    db.commit()

    print "Going into db!!! :)"
    id_row = db.execute("SELECT max(id_row) + 1 FROM kernel_results").fetchone()[0]
    id_row = [0, id_row][id_row != None]

    i = 0
    # ALL vs ALL
    for n1 in news:
        print n1
        for n2 in news:
            if n1 != n2:
                d = kernel(n1, n2, news)
                if d > 0:
                    db.execute("INSERT INTO kernel_results VALUES (%s, %s, %s, %s, %s, %s)" % (id_row, n1, n2, d, 0, kernel_words_length))
                    id_row += 1

        i += 1
        if i % 300 == 0:
            print "Commit"
            db.commit()

    db.commit()


print "End:", time.asctime()